/*
 *  Compiler.cpp
 *  vorpal
 *
 *  Created by Russell Webb on 20/03/08.
 *
 */

#include "Compiler.h"

namespace Vorpal{

int LocalNames::CreateLocal(string name){
	map< string, int >::iterator it = names.find(name);
	Assert(it == names.end());
	names[name] = numLocals;
	return numLocals++;
}

int LocalNames::CreateParam(string name){
	map< string, int >::iterator it = names.find(name);
	Assert(it == names.end());
	numParams += 1;
	names[name] = -numParams;
	return -numParams;
}

void LocalNames::Clear(){
	numParams = 0;
	numLocals = 0;
	names.clear();
}

bool LocalNames::Get(string name, int *value){
	map< string, int >::iterator it = names.find(name);
	if(it == names.end()){ return false; }
	
	if(value){ 
		*value = it->second; 
		if(*value < 0){
			*value = -(numParams + *value + 1);
		}
	}
	return true;
}

Compiler::Compiler() : 
Begin("Begin"),     Minus("Minus"),               Comment("Comment"), 
Decimal("Decimal"), Integer("Integer"),           Operator("Operator"), 
String("String"),   StringDouble("StringDouble"), Word("Word"), 
sourcePath("unknown"){
	
	Clear();
	GlobalName("sys");
	GlobalName("true");
	GlobalName("false");
	GlobalName("none");
	GlobalName("code");

	recordDebugInfo = traceOpcodes;
	
	static const UniqueToken keywords[] = {
		{ "has", HAS },
		{ "is", IS },
		{ "does", DOES },		
		{ "return", RETURN },
		{ "if", IF },
		{ "elseif", ELSEIF },
		{ "else", ELSE },		
		{ "while", WHILE },
		{ "break", BREAK },
		{ "method", METHOD },
		{ "new", NEW }, 
		{ "for", FOR },
		{ 0, ERROR },	// termination
	};

	static const UniqueToken special_ops[] = {
		{ "=", ASSIGN },
		{ 0, ERROR },	// termination
	};

	static const char whitespace[] = " \t";
	static const char operators[] = "-~!@$=%^&*-+\\|;<>?/:";

/*
 * The following code loads StateRules onto each LexerState's rule stack. Note that the () operators have been overloaded (See Lexer.cpp).
 */
	
	// Start
		Begin(end_of_file, NONE, StateRule::defer);
		Begin(whitespace, Begin, StateRule::discard);
		Begin("#", Comment, StateRule::discard);
		Begin("\n{}[]().,", SIMPLE);
		Begin("0123456789", Decimal);
		Begin(operators, Operator);
		Begin("'", String, StateRule::discard);
		Begin("\"", StringDouble, StateRule::discard);
		Begin(alpha, Word);
		Begin(otherwise, ERROR, StateRule::discard);

	// Comment
		Comment("\n", SIMPLE);
		Comment(end_of_file, NONE, StateRule::discard);
		Comment(otherwise, Comment, StateRule::discard);

	// Decimal
		Decimal(".", Integer);
		Decimal("0123456789", Decimal);
		Decimal(otherwise, NUMBER, StateRule::defer);

	// Integer
		Integer("0123456789", Integer);
		Integer(otherwise, NUMBER, StateRule::defer);

	// Operator
		Operator(operators, Operator);
		Operator(otherwise, OPERATOR, StateRule::defer);
		Operator(special_ops);

	// String 'text'
		String("'", STRING, StateRule::discard);
		String(end_of_file, STRING, StateRule::defer);
		String(otherwise, String);

	// StringDouble "text"
		StringDouble("\"", STRING, StateRule::discard);
		StringDouble(end_of_file, STRING, StateRule::defer);
		StringDouble(otherwise, StringDouble);
	
	// Word
		Word(alpha, Word);
		Word("0123456789_", Word);
		Word(otherwise, WORD, StateRule::defer);
		Word(keywords);
}

const char *Compiler::TokenLabel(int type){
	static const char *labels[NumTokenTypes] = {
		"ERROR",
		"NONE",
		"SIMPLE",
		"OPERATOR",
		"STRING",
		"WORD",
		"NUMBER",
		"ASSIGN",
		// keywords
		"HAS",
		"DOES",
		"IS",
		"RETURN",
		"IF",
		"ELSIF"
		"ELSE",
		"WHILE",
		"BREAK",
		"METHOD",
		"NEW",   
		"FOR",
	};
	Confirm(sizeof(labels)/sizeof(*labels) == NumTokenTypes);
	Assert(type < NumTokenTypes);
	return labels[type];
}

void Compiler::TestTokenizer(const char *path){
	bool trace = false;
	FileInMemory file(path);
	Tokenizer tokens;
	tokens.Process(file.Buffer(), strlen(file.Buffer()), Begin, trace);
	
	loop(i, 0, tokens.Size()){
		tokens[i].Debug();
	}	
}

/**
 * /brief insert implicit elements (anonymous method, self) into the script
 */
void Compiler::Preprocess(){
	// enclose code in an anonymous method(){ ... } if it isn't already enclosed
	uint32_t first = 0;
	while(first < tokens.Size() && tokens[first].type == SIMPLE && tokens[first].str == "\n"){ first += 1; }
	if(first == tokens.Size()){ 
		cout << "Empty file.\n";
		return; 
	}
	
	// a file that ends in the middle of a comment will leave a NONE token in the stream
	while(tokens[tokens.Size() - 1].type == NONE){ tokens.Remove(tokens.Size() - 1); }
	
	if(tokens[first].type != METHOD){
		tokens.Insert(0, Token(0, "{", 1, SIMPLE));
		tokens.Insert(0, Token(0, ")", 1, SIMPLE));
		tokens.Insert(0, Token(0, "(", 1, SIMPLE));
		tokens.Insert(0, Token(0, "method", 6, METHOD));
		int len = tokens.Size();
		int lineNum = tokens[len-1].lineNumber;
		tokens.Insert(len++, Token(lineNum, "}", 1, SIMPLE));
	}
	
	// insert implicit self
	for(uint32_t i = 1; i < tokens.Size(); ++i){
		if(tokens[i].type == SIMPLE && (tokens[i].str[0] == '[' || tokens[i].str[0] == '.')){
			if(tokens[i-1].Is(SIMPLE, "]") || tokens[i-1].Is(SIMPLE, ")") || tokens[i-1].type == WORD){
				continue;
			}
			if(tokens[i-1].type == WORD || tokens[i-1].type == NUMBER || tokens[i-1].type == STRING){
				continue;
			}
			tokens.Insert(i, Token(tokens[i].lineNumber, "self", 4, WORD));
			i += 1;
		}
	}		
}

bool Compiler::CompileString(const char *str){
	bool trace = false;
	Clear();
	tokens.Process(str, strlen(str), Begin, trace);
	
	Preprocess();
	
	try{
		return Method();
	}
	catch(Token errHere){
		cout << "Syntax error on line " << errHere.lineNumber << ": ";
		errHere.Debug();
		return false;
	}	
}

bool Compiler::Compile(const char *path){
	FileInMemory file(path);
	
	if(!file.Buffer()){
		cout << "File (" << path << ") was not loaded (permissions or file path may be incorrect).\n";
		return false;
	}
	
	sourcePath = path;
	return CompileString(file.Buffer());
}

void Compiler::Clear(){
	localNames.Clear();
	labelNum = 0;
	numArgs = 0; 
	numLocalsNeeded = 0;
	current = 0;
	tokens.Clear();
}

void Compiler::Need(bool match){
	const Token &next = tokens[current];
	if(!match){
		cout << "Unexpected syntax: " << tokens[current].str << endl;
		throw next;
	}
}

void Compiler::Need(int type, const char *simple){
	if(!Match(type, simple)){
		cout << "Unexpected syntax: " << tokens[current].str << ", expected " << TokenLabel(type) << endl;
		throw tokens[current];
	}
}

bool Compiler::FindSequenceOnLine(int type[], const char *simple[], uint32_t length){
	uint32_t here = current;
	while( here < tokens.Size() && (tokens[here].type != SIMPLE || tokens[here].str[0] != '\n') ){
		uint32_t matchLen = 0;
		for(uint32_t i = 0; i < length && matchLen == i && here + i < tokens.Size(); ++i){
			const Token &next = tokens[here + i];
			if(next.type == type[i] && (type[i] != SIMPLE || strchr(simple[i], next.str[0])) ){
				matchLen += 1;
			}
		}
		if(matchLen == length){
			return true;
		}
		here += 1;
	}
	return false;
}

bool Compiler::Match(int type, const char *simple){
	bool result = Compare(current, type, simple);
	if(result){
		current += 1;
	}
	return result;
}

void Compiler::Rewind(uint32_t numTokens){
	Assert(current >= numTokens);
	current -= numTokens;
}

bool Compiler::Previous(int type, const char *simple){
	return Compare(current - 1, type, simple);
}

bool Compiler::Compare(uint32_t index, int type, const char *simple){
	Assert(type != SIMPLE || simple != 0);
	if(index < tokens.Size()){
		const Token &next = tokens[index];
		if(next.type == type && (type != SIMPLE || strchr(simple, next.str[0])) ){
			if(false){ // here for debugging
				printf("match: ");
				next.Debug();
			}
			return true;
		}
	}
	return false;
}


void Compiler::AnyNewlines(){
	while(Match(SIMPLE, "\n")){ }
}

// ---------------------------------------------------------------------------
// Following methods implement the recursive decent parser
// ---------------------------------------------------------------------------
/*
	The recursive decent parser is composed of methods parsing 
	different parts of the grammar.
	
	Each method returns a boolean indicating success and should 
	have no side-effects on failure (meaning the parse stream 
	is still in the current location).  
	
	The ATTEMPT, NEED, and SUCCESS macros are designed to help 
	write these parser methods.  Start such methods with ATTEMPT 
	and use NEED to require matches from other parts.  An error 
	string is supplied to need and is printed on failure.  A error 
	string of "" is ignored.  Return true at the end via the 
	SUCCESS macro so that other bookeeping can be done.
 
	Basically, when you ATTEMPT to reach SUCCESS you will NEED things.
 
	FIX: Failure to meet a NEED should also backtrack the Assembler, 
	but currently the assembler lacks that ability which means we 
	have to do more look-ahead in the parser.
*/

void Compiler::ReportParseError(const char *errString){
	if(*errString){
		if(current == tokens.Size() && tokens.Size() > 0){
			current -= 1;
		}
		if(current >= tokens.Size()){
			cout << "Error at end of file." << endl;
			cout << errString << endl;
			return;
		}
		
		// find the start of the line or file
		uint32_t here = current;
		if(tokens[here].type == SIMPLE && tokens[here].str[0] == '\n' && here > 0){ 
			// backup so we show the previous line
			here -= 1;
		}
		while(true){
			if(tokens[here].type == SIMPLE && tokens[here].str[0] == '\n'){ 
				here += 1;
				break; 
			}
            if(here != 0){
                here -= 1;
            }
            else{
                break;      
            }
		}
		
		// print tokens up to error
		char buffer[200];
		sprintf(buffer, "Line %u:  ", tokens[here].lineNumber);
		uint32_t charsToError = strlen(buffer);
		cout << buffer;
		while(here <= current && here < tokens.Size()){
			charsToError += tokens[here].str.length() + 1;
			cout << tokens[here].str << " ";
			here += 1;
		}
		charsToError -= tokens[current].str.length() + 1;
		
		// print error message
		printf("\n");
		loop(i, 0, charsToError){
			printf(" ");
		}
		printf("^ here: %s\n", errString);
		
		// stop compiling
		throw tokens[current]; 
	}
}

#define TRACE(prefix) if(traceParser && current < tokens.Size()){ printf("%s%s on line %u\n", prefix, __func__, tokens[current].lineNumber); }

#define ATTEMPT() TRACE("\t?: ") uint32_t initial = current 
#define SUCCESS() TRACE("success: ") return true
#define NEED(cond, errStr) if(!(cond)){ ReportParseError(errStr); current = initial; return false; }

bool Compiler::Method(){
ATTEMPT();
	localNames.Clear();
	numArgs = 1; // implicit self arg
	localNames.CreateParam(string("self"));
	
	AnyNewlines();
	NEED(Match(METHOD), "");
	NEED(Parameters(), "parameter formatting is invalid");
	NEED(Brace("{"), "missing open brace");		
	bool lastWasReturn = false;
	NEED(CodeToBrace(&lastWasReturn), "code block invalid");
	if(!lastWasReturn){
		AsmReturn(/*returnTop*/ false);
	}
	AsmEndMethod(numLocalsNeeded);
SUCCESS();
}

bool Compiler::SubMethod(){
ATTEMPT();
	static int labelBase = 0;
	labelBase += 1000;
	// FIX: seems wasteful the have Method() and Submethod(), but they do function differently
	NEED(Match(METHOD), "");
	// create a completely new compiler and compile the method
	Compiler subCompiler(*this);
	subCompiler.localNames.Clear();
	subCompiler.numLocalsNeeded = 0;
	subCompiler.assemble = Assembler(); // FIX: this is a bit of a hack to get a fresh assembler that doesn't know about existing labels
	subCompiler.labelNum = labelBase;
	subCompiler.current -= 1;
	try{
		NEED(subCompiler.Method(), "error in method");
	}
	catch(...){
		ReportParseError("unknown error in method");
		throw;
	}
	DeclareArray(code, Cell, subCompiler.Size());
	subCompiler.Get(code, subCompiler.Size());
	AsmCode(code, subCompiler.Size());
	current = subCompiler.current - 1; 
SUCCESS();
}
	
bool Compiler::Comma(){
ATTEMPT();
	NEED(Simple(","), "");
	AnyNewlines();
SUCCESS();	
}

bool Compiler::VariableLengthParam(uint &paramIndex){
ATTEMPT();
	NEED(Simple("["), "");
	NEED(paramIndex == 0, "only one variable length parameter per method");
	NEED(Simple("]"), "no closing ']'");
	paramIndex = localNames.NumParams() - 1;
	Assert(paramIndex > 0);
SUCCESS();	
}
	
bool Compiler::Parameters(){
uint paramIndex = 0;
ATTEMPT();
	NEED(OpenParen(), "");
	if(Parameter()){
		VariableLengthParam(paramIndex);
		while(Comma()){
			NEED(Parameter(), "no parameter after comma");
			VariableLengthParam(paramIndex);
		}
	}
	NEED(CloseParen(), "missing ')'");
	AsmStartMethod(numArgs, paramIndex);
SUCCESS();
}

bool Compiler::PreviousSimple(const char *options){
ATTEMPT();
	NEED(Previous(SIMPLE, options), "");
SUCCESS();
}

bool Compiler::Simple(const char *options){
ATTEMPT();
	NEED(Match(SIMPLE, options), "");
SUCCESS();
}

bool Compiler::Parameter(){
ATTEMPT();
	NEED(Match(WORD), "");
	AsmParameter(current - 1);
	numArgs += 1;
SUCCESS();
}

bool Compiler::Brace(const char *which){
ATTEMPT();
	AnyNewlines();
	NEED(Simple(which), "");
	AnyNewlines();	
SUCCESS();
}
	
bool Compiler::OpenParen(){
ATTEMPT();
	NEED(Simple("("), "");
	AnyNewlines();	
SUCCESS();
}
	
bool Compiler::CloseParen(){
ATTEMPT();
	AnyNewlines();
	NEED(Simple(")"), "");
SUCCESS();
}

bool Compiler::CodeToBrace(bool *lastWasReturn){
ATTEMPT();
	bool lastReturn = false;
	while(!Brace("}")){
		lastReturn = false;
		if     (  Assignment() ){ }
		else if(  Expression() ){ }
		else if(          If() ){ }
		else if(       While() ){ }
		else if(         For() ){ }
		else if(      Return() ){ 			
			lastReturn = true;		
		}
		else{ 
			if(current < tokens.Size()){
				NEED(false, "unrecognized code structure");
			}
			else{
				current = tokens.Size() - 1;
				NEED(false, "no closing brace for method before end of file");
			}
		}
		AnyNewlines();
	}	
	AnyNewlines();	
	if(lastReturn && lastWasReturn != 0){
		*lastWasReturn = lastReturn;
	}
SUCCESS();
}

bool Compiler::While(){
ATTEMPT();
	char condTest[80];
	char afterLoop[80];
	UniqueLabel(condTest);
	UniqueLabel(afterLoop);
	
	NEED(Match(WHILE), "");
	NEED(OpenParen(), "missing '('");
	AsmLabel(condTest);
	break_to_here.push_back(afterLoop);
	
	NEED(Expression(), "a non-empty expression needed to control while loop");
	NEED(CloseParen(), "missing ')'");
	
	AsmJumpFalse(afterLoop);
	
	NEED(Brace("{"), "missing open brace");
	NEED(CodeToBrace(), "error in code");
	AsmJump(condTest);
	AsmLabel(afterLoop);
	break_to_here.pop_back();
	
SUCCESS();
}

bool Compiler::For(){
ATTEMPT();
	char condTest[80];
	char afterLoop[80];
	char incrementer[80];
	char codeBody[80];
	UniqueLabel(condTest);
	UniqueLabel(afterLoop);
	UniqueLabel(incrementer);
	UniqueLabel(codeBody);
	
	NEED(Match(FOR), "");
	NEED(OpenParen(), "missing '('");
	//find initialisation
	NEED(Assignment(), "a non-empty initialisation assignment needed to set up for loop");
	NEED(Comma(), "missing ','");
	break_to_here.push_back(afterLoop);
	
	//find condition test
	AsmLabel(condTest);
	NEED(Expression(), "a non-empty expression needed to control for loop");
	NEED(Comma(), "missing ','");
	AsmJumpFalse(afterLoop);
	
	AsmJump(codeBody);
	
	//find incrementation
	AsmLabel(incrementer);
	NEED(Assignment(), "error in code");
	NEED(CloseParen(), "missing ')'");
	AsmJump(condTest);
	
	//find code body
	AsmLabel(codeBody);
	NEED(Brace("{"), "missing open brace");
	NEED(CodeToBrace(), "error in code");
	AsmJump(incrementer);
	
	AsmLabel(afterLoop);
	break_to_here.pop_back();
	
SUCCESS();
}

bool Compiler::Break(){
ATTEMPT();
    NEED(Match(BREAK), "");
    if(break_to_here.size() > 0){
        AsmJump(break_to_here.back());
    }
    else{
        NEED(false , "trying to break while outside of loop");
    }
SUCCESS();
}

bool Compiler::Return(){
ATTEMPT();
	NEED(Match(RETURN), "");
	NEED(OpenParen(), "missing '('");
	NEED(Expression(), "return needs a valid expression");
	NEED(CloseParen(), "missing ')'");
	AnyNewlines();
	AsmReturn(/*returnTop*/ true);
SUCCESS();
}

bool Compiler::If(){
ATTEMPT();
	char onFalse[80];
	char overallEnd[80];
	UniqueLabel(onFalse);
	UniqueLabel(overallEnd);
	
	NEED(Match(IF), "");
	NEED(OpenParen(), "missing '('");
	NEED(Expression(), "if needs a condition expression");
	NEED(CloseParen(), "missing ')'");
	AsmJumpFalse(onFalse);
	
	NEED(Brace("{"), "missing '{'");
	NEED(CodeToBrace(), "code block invalid");
	AsmJump(overallEnd);
	
	while(ElseIf(onFalse, overallEnd)){ }
	
	AsmLabel(onFalse);
	Else(); // optional
	AsmLabel(overallEnd);
SUCCESS();
}

bool Compiler::ElseIf(char *startLabel, const char *finalLabel){
ATTEMPT();
	char endElseIf[80];
	UniqueLabel(endElseIf);
	
	NEED(Match(ELSEIF), "");
	NEED(OpenParen(), "missing '('");
	AsmLabel(startLabel);
	UniqueLabel(startLabel); // refresh the label for the next jump in the chain
	NEED(Expression(), "elseif needs a condition expression");
	NEED(CloseParen(), "missing ')'");
	AsmJumpFalse(endElseIf);
	
	NEED(Brace("{"), "missing '{'");
	NEED(CodeToBrace(), "code block invalid");
	AsmJump(finalLabel);
	AsmLabel(endElseIf);
SUCCESS();
}

bool Compiler::Else(){
ATTEMPT();
	NEED(Match(ELSE), "");
	NEED(Brace("{"), "missing '{'");
	NEED(CodeToBrace(), "code block invalid");
SUCCESS();
}

bool Compiler::DotMessage(){
ATTEMPT();
	NEED(Match(WORD), "invalid name of message");
	uint32_t args = 0;
	NEED(Arguments(&args), "missing arguments for message");
	args += 1; // value on stack is first argument
	AsmMessage(args, initial);
SUCCESS();
}

bool Compiler::Arguments(uint32_t *numArgs){
ATTEMPT();
	*numArgs = 0;
	NEED(OpenParen(), "");
	if(Expression(false)){
		Match(Simple("}")); // skip ending brace of method arguments
		*numArgs += 1;
		while(Comma()){
			*numArgs += 1;
			NEED(Expression(false), "argument required after comma");
			Match(Simple("}")); // skip ending brace of method arguments
		}
	}
	NEED(CloseParen(), "missing ')'");
SUCCESS();
}

bool Compiler::Number(){
ATTEMPT();
	bool negate = false;
	if(Match(OPERATOR) && tokens[current - 1].str == "-"){
		negate = true;
	}
	NEED(Match(NUMBER), "");
	AsmNumber(current - 1, negate);
SUCCESS();
}

bool Compiler::Assignment(){
ATTEMPT();
	NEED(LocalAssignment() || SlotAssignment(), "");
SUCCESS();
}

bool Compiler::DynamicSlot(){
ATTEMPT();
	NEED(Match(SIMPLE, "["), "");
	NEED(Expression(/*dropAtNewline*/ false), "invalid expression in dynamic slot");
	NEED(Simple("]"), "missing ']'");
SUCCESS();	
}

bool Compiler::LocalAssignment(){
ATTEMPT();
	NEED(Match(WORD) && Match(ASSIGN), "");
	NEED(Expression(/*dropAtNewline*/ false), "invalid expression for local assignment");
	AsmAssignLocal(initial);
SUCCESS();
}

bool Compiler::SlotSetLast(uint32_t *setIndex, bool *dynamicAssign){
ATTEMPT();
	NEED(Simple(".") || Simple("["), "");
	if(PreviousSimple("[")){
		Rewind(1);
	}
	NEED(Match(WORD) || Match(OPERATOR) || DynamicSlot(), "missing slot description");
	if(Previous(WORD) || Previous(OPERATOR)){
		if(Simple(".") || Simple("[")){
			Rewind(1);
			AsmGetObject(current - 1); // FIX: should call AsmRoutine here
		}
		else{
			*setIndex = current - 1;
		}
	}
	else{
		if(Simple(".") || Simple("[")){
			Rewind(1);
			assemble.ObjectGet(0); // FIX: should call AsmRoutine here
		}
		else{
			*dynamicAssign = true;
		}			
	}
SUCCESS();	
}

bool Compiler::SlotAssignment(){
ATTEMPT();
	int types[] = { ASSIGN };
	const char *simple[] = { "" };
	NEED(FindSequenceOnLine(types, simple, sizeof(types)/sizeof(*types)), "");	
	
	NEED(Match(WORD) && (Simple(".") || Simple("[")), "");
	Rewind(1);
	
	if(globalNames.find(tokens[initial].str) != globalNames.end() && globalNames[tokens[initial].str]){ 
		// globals are a special exception
		AsmGlobal(tokens[initial].str.c_str());
	}
	else{
		AsmIdentifier(initial);
	}		
	uint32_t setIndex = 0;
	bool dynamicAssign = false;
	while(SlotSetLast(&setIndex, &dynamicAssign)){ }
	NEED(setIndex != 0 || dynamicAssign, "invalid final slot");
	NEED(Match(ASSIGN), "assignment not found at end of slot identifiers");
	NEED(Expression(/*dropAtNewline*/ false), "invalid expression for slot assignment");
	if(dynamicAssign){
		assemble.ObjectSet(0); // TODO: should call AsmRoutine here
	}
	else{
		AsmSetObject(setIndex);
	}
	AsmDrop();
SUCCESS();
}

bool Compiler::TableDefinition(){ // TODO: should not require comma at end of list, instead require } if comma is missing
ATTEMPT();
	NEED(Simple("{"), "");
	AnyNewlines();
	while(!Simple("}")){
		NEED(Expression(false, false), "invalid expression");
		Match(Simple("}"));
		if(Match(ASSIGN)){ // slot assignment using =
			AnyNewlines();			
			NEED(Expression(false), "invalid expression");
			Match(Simple("}")); // TODO: closing bracket of method and tables should be consumed already
			NEED(Simple(","), "expected comma");
			AnyNewlines();
			assemble.ObjectSet(0); // TODO: should call AsmRoutine here
		}
		else{ // append value
			NEED(Simple(","), "expected comma");
			AnyNewlines();
			assemble.Send(2, "append"); // TODO: should call AsmRoutine here
		}
	}
SUCCESS();
}

bool Compiler::Expression(bool dropAtNewline, bool expectNormalEnd){
ATTEMPT();
	if(Break()){
	    NEED(Simple("\n)}],"), "expression should end at newline, ')', '}', ']', or ','");
	    Rewind(1);
	}
	else{
	    NEED(SingleUnit() || Number(), "");
	    while(true){
	    	if(Match(OPERATOR)){
	    		uint32_t opIndex = current - 1;
				AnyNewlines();
	    		NEED(SingleUnit() || Number(), "operator not followed by a parameter");
	    		AsmBinary(opIndex);
	    	}
	    	else if(Match(SIMPLE, ".")){
	    		NEED(DotMessage(), "invalid message after dot operator\n");
	    	}
	    	else if(TableDefinition()){ 
				break;
			}
	    	else{
	    		break;
	    	}
	    }
		if(expectNormalEnd){
			NEED(Simple("\n)}],"), "expression should end at newline, ')', '}', ']', or ','");
			Rewind(1);
    	}
		if(Simple("\n") && dropAtNewline){
    		AsmDrop();
    	}
	}
SUCCESS();
}

bool Compiler::New(){
ATTEMPT();
	NEED(Match(NEW), "");
	uint32_t numArgs;
	NEED(Arguments(&numArgs), "missing arguments for new()");
	AsmObject(numArgs);	
SUCCESS();
}
	
bool Compiler::SingleUnit(){
ATTEMPT();
	uint32_t unaryIndex = 0;
	if(Match(OPERATOR)){
		unaryIndex = current - 1;
	}
	if(OpenParen()){
		NEED(Expression(false), "expression needed");
		NEED(CloseParen(), "missing ')'");
	}
	else if(SlotIdentifier()){ }
	else if(Match(STRING)){
		AsmString(current - 1);
	}
	else if(SubMethod()){ }
	else if(New()){ }
	else{
		NEED(Match(WORD), "");
		AsmIdentifier(current - 1);
	}
	if(unaryIndex){
		AsmUnary(unaryIndex);
	}
SUCCESS();
}

bool Compiler::SlotGet(){
ATTEMPT();
	NEED(Match(SIMPLE, ".") || Match(SIMPLE, "["), "");
	if(tokens[current - 1].str == "["){
		Rewind(1);
	}
	NEED(Match(WORD) || Match(OPERATOR) || DynamicSlot(), "missing slot description");
	if(Previous(WORD) || Previous(OPERATOR)){
		if(Match(SIMPLE, "(")){
			Rewind(2);
			NEED(DotMessage(), "message incomplete");
		}
		else{
			AsmGetObject(current - 1);
		}
	}
	else{
		if(Match(SIMPLE, "(")){
			Rewind(1);
			uint32_t numArgs;
			NEED(Arguments(&numArgs), "invalid arguments");
			assemble.Send(numArgs + 1, 0);  // FIX: should call AsmRoutine here
		}
		else{
			assemble.ObjectGet(0); // FIX: should call AsmRoutine here
		}
	}
SUCCESS();
}

bool Compiler::SlotIdentifier(){
ATTEMPT();
	NEED(Match(WORD), "");
	
	if(globalNames.find(tokens[initial].str) != globalNames.end() && globalNames[tokens[initial].str]){ 
		// globals are a special exception
		AsmGlobal(tokens[initial].str.c_str());
	}
	else{
		AsmIdentifier(initial);
	}		
	
	while(SlotGet()){ }
SUCCESS();
}

// ---------------------------------------------------------------------------
// Following methods call the Assembler to do the assembly during parsing
// ---------------------------------------------------------------------------
void Compiler::UniqueLabel(char *label){
	sprintf(label, "L%u", labelNum++);
}

void Compiler::AsmLabel(const char *label){
	if(traceAsm){ printf("%s:\n", label); }
	assemble.Label(label);
	AsmRecordDebugInfo();
}

void Compiler::AsmJump(const char *label){
	if(traceAsm){ printf("    JUMP %s\n\n", label); }
	assemble.Jump(label);
	AsmRecordDebugInfo();
}

void Compiler::AsmJumpFalse(const char *label){
	if(traceAsm){ printf("    JUMP_F %s\n\n", label); }
	assemble.JumpFalse(label);
	AsmRecordDebugInfo();
}

void Compiler::AsmNumber(uint32_t tokenIndex, bool negate){
	if(traceAsm){ printf("    NUM ");  tokens[tokenIndex].Print(); }	
	double value = atof(tokens[tokenIndex].str.c_str());
	if(negate){ value = -value; }
	assemble.Number(value);
	AsmRecordDebugInfo();
}

void Compiler::AsmDrop(){
	if(traceAsm){ puts("    DROP "); }
	assemble.Drop();
	AsmRecordDebugInfo();
}

void Compiler::AsmParameter(uint32_t index){
	if(localNames.Get(tokens[index].str)){
		current = index;
		ReportParseError("duplicate argument name");
		throw tokens[index];
	}
	int location = localNames.CreateParam(tokens[index].str);
	if(traceAsm){ printf("    ARG[%i] ", location); tokens[index].Print(); }
	AsmRecordDebugInfo();
}

void Compiler::AsmStartMethod(uint32_t numArgs, uint32_t varParamIndex){
	if(traceAsm){ printf("    NUM_ARGS %u\n", numArgs); }
	assemble.StartMethod(numArgs, varParamIndex);
	AsmRecordDebugInfo();
}

void Compiler::AsmEndMethod(uint32_t numLocals){
	if(traceAsm){ printf("    NUM_LOCALS %u\n", numLocals); }
	assemble.EndMethod(numLocals);
	AsmRecordDebugInfo();
}

void Compiler::AsmReturn(bool returnTop){
	if(traceAsm){ puts("    RETURN"); }
	assemble.Return(returnTop ? -numArgs : 0);
	AsmRecordDebugInfo();
}

void Compiler::AsmAssignLocal(uint32_t tokenIndex){
	int localIndex = 0;
	if(localNames.Get(tokens[tokenIndex].str, &localIndex)){
		if(traceAsm){ printf("    SET_LOCAL[%i] ", localIndex); tokens[tokenIndex].Print(); }
	}
	else{
		localIndex = localNames.CreateLocal(tokens[tokenIndex].str);		
		if(traceAsm){ printf("    CREATE_LOCAL[%i] ", localIndex); tokens[tokenIndex].Print(); }
		numLocalsNeeded += 1;
	}
	assemble.Set(localIndex);
	AsmRecordDebugInfo();
}

void Compiler::AsmIdentifier(uint32_t tokenIndex){
	int localIndex = 0;
	if(localNames.Get(tokens[tokenIndex].str, &localIndex)){
		if(traceAsm){ printf("    GET_LOCAL[%i] ", localIndex);  tokens[tokenIndex].Print(); }
		assemble.Get(localIndex);
	}
	else{
		current = tokenIndex;
		ReportParseError("unknown identifier");
		throw tokens[tokenIndex];
	}
	AsmRecordDebugInfo();
}

void Compiler::AsmString(uint32_t tokenIndex){
	if(traceAsm){ printf("    STR: %s\n", tokens[tokenIndex].str.c_str()); }
	string convert = tokens[tokenIndex].str;
	int here = 0;
	loop(i, 0, convert.length()){
		if(convert[i] == '\\'){
			i += 1;
			switch(convert[i]){
				case 'n':  convert[here++] = '\n'; break;
				case 't':  convert[here++] = '\t'; break;
				case '\\': convert[here++] = '\\'; break;		
				default:
					i -= 1;
					convert[here++] = convert[i];
			}
		}
		else{
			convert[here++] = convert[i];
		}
	}
	convert.resize(here);
	assemble.String(convert.c_str());	
	AsmRecordDebugInfo();
}

void Compiler::AsmMessage(uint32_t numArgs, uint32_t messageTokenIndex){
	if(traceAsm){ printf("    MESSAGE(%u) ", numArgs);  tokens[messageTokenIndex].Print(); }
	assemble.Send(numArgs, tokens[messageTokenIndex].str.c_str());
	AsmRecordDebugInfo();
}

void Compiler::AsmBinary(uint32_t tokenIndex){
	if(traceAsm){ printf("    BINOP ");  tokens[tokenIndex].Print(); }
	assemble.Send(2, tokens[tokenIndex].str.c_str());
	AsmRecordDebugInfo();
}

void Compiler::AsmUnary(uint32_t tokenIndex){
	if(traceAsm){ printf("    UNARY ");  tokens[tokenIndex].Print(); }
	assemble.Send(1, tokens[tokenIndex].str.c_str());
	AsmRecordDebugInfo();
}

void Compiler::AsmObject(uint32_t numArgs){
    if(traceAsm){ printf("    CREATE_OBJECT\n"); }
    assemble.Object(numArgs);
	AsmRecordDebugInfo();
}

void Compiler::AsmSetObject(uint32_t keyTokenIndex){
	if(traceAsm){ printf("    OBJECT_SET "); tokens[keyTokenIndex].Print(); }
	assemble.ObjectSet(tokens[keyTokenIndex].str.c_str());		
	AsmRecordDebugInfo();
}

void Compiler::AsmGetObject(uint32_t keyTokenIndex){
	if(traceAsm){ printf("    OBJECT_GET "); tokens[keyTokenIndex].Print(); }
	assemble.ObjectGet(tokens[keyTokenIndex].str.c_str());		
	AsmRecordDebugInfo();
}

void Compiler::AsmGlobal(const char *name){
	if(traceAsm){ printf("    Symbol: %s\n", name); }
	assemble.Symbol(name);
	AsmRecordDebugInfo();
}

void Compiler::AsmCode(const Cell *code, uint32_t codeSize){
	if(traceAsm){ printf("    CODE[%u]\n", codeSize); }
	assemble.Code(code, codeSize);
	AsmRecordDebugInfo();
}

void Compiler::AsmRecordDebugInfo(){
	if(recordDebugInfo && current < tokens.Size()){ 
		assemble.SetDebugLineInfo(sourcePath, tokens[current].lineNumber);
	}
}
	
}

